{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文本分类实例"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step1 导入相关包"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:07:50.498112Z",
     "start_time": "2025-09-07T11:07:50.489090Z"
    }
   },
   "source": [
    "from datasets import load_dataset\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "torch.cuda.set_device(0)\n",
    "torch.cuda.empty_cache()"
   ],
   "outputs": [],
   "execution_count": 195
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step2 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:07:52.712172Z",
     "start_time": "2025-09-07T11:07:51.940713Z"
    }
   },
   "source": [
    "dataset = load_dataset(\"csv\", data_files = \"./ChnSentiCorp_htl_all.csv\", split = \"train\")\n",
    "dataset = dataset.filter(lambda x: x[\"review\"] is not None)\n",
    "dataset"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'review'],\n",
       "    num_rows: 7765\n",
       "})"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 196
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step3 划分数据集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:07:56.939702Z",
     "start_time": "2025-09-07T11:07:56.927536Z"
    }
   },
   "source": [
    "datasets = dataset.train_test_split(test_size = 0.1)\n",
    "datasets"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 197
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step4 数据集预处理"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:08:00.812235Z",
     "start_time": "2025-09-07T11:07:58.881732Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"hfl/chinese-macbert-large\")\n",
    "\n",
    "\n",
    "def process_function(examples):\n",
    "\ttokenized_examples = tokenizer(examples[\"review\"], max_length = 32, truncation = True, padding = \"max_length\")\n",
    "\ttokenized_examples[\"labels\"] = examples[\"label\"]\n",
    "\treturn tokenized_examples\n",
    "\n",
    "\n",
    "tokenized_datasets = datasets.map(process_function, batched = True, remove_columns = datasets[\"train\"].column_names)\n",
    "tokenized_datasets"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 198
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step5 创建模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:08:04.477076Z",
     "start_time": "2025-09-07T11:08:02.731186Z"
    }
   },
   "source": [
    "def ensure_weights_contiguous(model):\n",
    "\t# model.named_parameters(): 获取模型中所有命名参数\n",
    "\tfor name, param in model.named_parameters():\n",
    "\t\t# param.is_contiguous(): 检查参数在内存中是否连续存储\n",
    "\t\tif not param.is_contiguous():\n",
    "\t\t\tprint(f\"Making {name} contiguous.\")\n",
    "\t\t\t# param.data.contiguous(): 创建连续存储版本并替换原数据\n",
    "\t\t\tparam.data = param.data.contiguous()\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"hfl/chinese-macbert-large\")\n",
    "# 调用函数确保模型权重连续性\n",
    "ensure_weights_contiguous(model)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at hfl/chinese-macbert-large and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making bert.encoder.layer.0.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.0.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.0.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.0.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.0.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.0.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.1.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.1.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.1.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.1.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.1.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.1.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.2.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.2.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.2.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.2.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.2.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.2.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.3.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.3.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.3.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.3.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.3.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.3.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.4.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.4.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.4.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.4.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.4.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.4.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.5.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.5.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.5.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.5.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.5.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.5.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.6.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.6.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.6.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.6.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.6.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.6.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.7.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.7.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.7.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.7.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.7.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.7.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.8.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.8.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.8.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.8.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.8.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.8.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.9.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.9.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.9.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.9.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.9.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.9.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.10.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.10.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.10.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.10.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.10.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.10.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.11.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.11.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.11.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.11.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.11.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.11.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.12.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.12.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.12.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.12.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.12.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.12.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.13.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.13.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.13.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.13.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.13.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.13.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.14.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.14.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.14.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.14.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.14.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.14.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.15.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.15.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.15.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.15.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.15.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.15.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.16.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.16.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.16.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.16.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.16.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.16.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.17.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.17.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.17.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.17.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.17.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.17.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.18.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.18.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.18.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.18.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.18.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.18.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.19.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.19.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.19.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.19.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.19.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.19.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.20.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.20.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.20.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.20.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.20.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.20.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.21.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.21.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.21.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.21.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.21.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.21.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.22.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.22.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.22.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.22.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.22.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.22.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.23.attention.self.query.weight contiguous.\n",
      "Making bert.encoder.layer.23.attention.self.key.weight contiguous.\n",
      "Making bert.encoder.layer.23.attention.self.value.weight contiguous.\n",
      "Making bert.encoder.layer.23.attention.output.dense.weight contiguous.\n",
      "Making bert.encoder.layer.23.intermediate.dense.weight contiguous.\n",
      "Making bert.encoder.layer.23.output.dense.weight contiguous.\n",
      "Making bert.pooler.dense.weight contiguous.\n"
     ]
    }
   ],
   "execution_count": 199
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step6 创建评估函数"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:08:05.903235Z",
     "start_time": "2025-09-07T11:08:05.877649Z"
    }
   },
   "source": [
    "import evaluate\n",
    "\n",
    "\n",
    "# 如果网络不太好，也可以使用本地加载的方式\n",
    "acc_metric = evaluate.load(\"./metric_accuracy.py\")\n",
    "f1_metirc = evaluate.load(\"./metric_f1.py\")"
   ],
   "outputs": [],
   "execution_count": 200
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:08:07.553468Z",
     "start_time": "2025-09-07T11:08:07.549130Z"
    }
   },
   "source": [
    "def eval_metric(eval_predict):\n",
    "\tpredictions, labels = eval_predict\n",
    "\tpredictions = predictions.argmax(axis = -1)\n",
    "\tacc = acc_metric.compute(predictions = predictions, references = labels)\n",
    "\tf1 = f1_metirc.compute(predictions = predictions, references = labels)\n",
    "\tacc.update(f1)\n",
    "\treturn acc"
   ],
   "outputs": [],
   "execution_count": 201
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step7 创建TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T12:48:19.812905Z",
     "start_time": "2025-09-07T12:48:19.675926Z"
    }
   },
   "source": [
    "train_args = TrainingArguments(\n",
    "\toutput_dir = \"./checkpoints\",  # 模型和日志输出目录\n",
    "\tper_device_train_batch_size = 64,  # 训练时每个设备的批次大小\n",
    "\tper_device_eval_batch_size = 128,  # 评估时每个设备的批次大小\n",
    "\tgradient_accumulation_steps = 4,  # 梯度累积步数\n",
    "\tgradient_checkpointing = True,  # 启用梯度检查点以节省内存\n",
    "\tgradient_checkpointing_kwargs = {\"use_reentrant\": False},  # 推荐设置\n",
    "\toptim = \"adafactor\",  # 使用Adafactor优化器\n",
    "\tsave_steps = 1000,  # 每1000步保存一次模型\n",
    "\tfp16 = True,  # 使用16位浮点数训练\n",
    "\tlogging_steps = 100,  # 每100步记录一次日志\n",
    "\tnum_train_epochs = 3,  # 训练3个epoch\n",
    "\teval_strategy = \"epoch\",  # 每个epoch结束后进行评估\n",
    "\tsave_strategy = \"epoch\",  # 每个epoch结束后保存模型\n",
    "\tsave_total_limit = 3,  # 最多保存3个检查点\n",
    "\tlearning_rate = 2e-5,  # 学习率\n",
    "\tweight_decay = 0.01,  # 权重衰减系数\n",
    "\tmetric_for_best_model = \"f1\",  # 以F1分数作为最佳模型选择标准\n",
    "\tload_best_model_at_end = True,  # 训练结束后加载最佳模型\n",
    ")\n",
    "train_args"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TrainingArguments(\n",
       "_n_gpu=1,\n",
       "accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False},\n",
       "adafactor=False,\n",
       "adam_beta1=0.9,\n",
       "adam_beta2=0.999,\n",
       "adam_epsilon=1e-08,\n",
       "auto_find_batch_size=False,\n",
       "batch_eval_metrics=False,\n",
       "bf16=False,\n",
       "bf16_full_eval=False,\n",
       "data_seed=None,\n",
       "dataloader_drop_last=False,\n",
       "dataloader_num_workers=0,\n",
       "dataloader_persistent_workers=False,\n",
       "dataloader_pin_memory=True,\n",
       "dataloader_prefetch_factor=None,\n",
       "ddp_backend=None,\n",
       "ddp_broadcast_buffers=None,\n",
       "ddp_bucket_cap_mb=None,\n",
       "ddp_find_unused_parameters=None,\n",
       "ddp_timeout=1800,\n",
       "debug=[],\n",
       "deepspeed=None,\n",
       "disable_tqdm=False,\n",
       "dispatch_batches=None,\n",
       "do_eval=True,\n",
       "do_predict=False,\n",
       "do_train=False,\n",
       "eval_accumulation_steps=None,\n",
       "eval_delay=0,\n",
       "eval_do_concat_batches=True,\n",
       "eval_on_start=False,\n",
       "eval_steps=None,\n",
       "eval_strategy=IntervalStrategy.EPOCH,\n",
       "eval_use_gather_object=False,\n",
       "evaluation_strategy=None,\n",
       "fp16=True,\n",
       "fp16_backend=auto,\n",
       "fp16_full_eval=False,\n",
       "fp16_opt_level=O1,\n",
       "fsdp=[],\n",
       "fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},\n",
       "fsdp_min_num_params=0,\n",
       "fsdp_transformer_layer_cls_to_wrap=None,\n",
       "full_determinism=False,\n",
       "gradient_accumulation_steps=4,\n",
       "gradient_checkpointing=True,\n",
       "gradient_checkpointing_kwargs={'use_reentrant': False},\n",
       "greater_is_better=True,\n",
       "group_by_length=False,\n",
       "half_precision_backend=auto,\n",
       "hub_always_push=False,\n",
       "hub_model_id=None,\n",
       "hub_private_repo=False,\n",
       "hub_strategy=HubStrategy.EVERY_SAVE,\n",
       "hub_token=<HUB_TOKEN>,\n",
       "ignore_data_skip=False,\n",
       "include_inputs_for_metrics=False,\n",
       "include_num_input_tokens_seen=False,\n",
       "include_tokens_per_second=False,\n",
       "jit_mode_eval=False,\n",
       "label_names=None,\n",
       "label_smoothing_factor=0.0,\n",
       "learning_rate=2e-05,\n",
       "length_column_name=length,\n",
       "load_best_model_at_end=True,\n",
       "local_rank=0,\n",
       "log_level=passive,\n",
       "log_level_replica=warning,\n",
       "log_on_each_node=True,\n",
       "logging_dir=./checkpoints\\runs\\Sep07_20-48-19_yumeko,\n",
       "logging_first_step=False,\n",
       "logging_nan_inf_filter=True,\n",
       "logging_steps=100,\n",
       "logging_strategy=IntervalStrategy.STEPS,\n",
       "lr_scheduler_kwargs={},\n",
       "lr_scheduler_type=SchedulerType.LINEAR,\n",
       "max_grad_norm=1.0,\n",
       "max_steps=-1,\n",
       "metric_for_best_model=f1,\n",
       "mp_parameters=,\n",
       "neftune_noise_alpha=None,\n",
       "no_cuda=False,\n",
       "num_train_epochs=3,\n",
       "optim=OptimizerNames.ADAFACTOR,\n",
       "optim_args=None,\n",
       "optim_target_modules=None,\n",
       "output_dir=./checkpoints,\n",
       "overwrite_output_dir=False,\n",
       "past_index=-1,\n",
       "per_device_eval_batch_size=128,\n",
       "per_device_train_batch_size=64,\n",
       "prediction_loss_only=False,\n",
       "push_to_hub=False,\n",
       "push_to_hub_model_id=None,\n",
       "push_to_hub_organization=None,\n",
       "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
       "ray_scope=last,\n",
       "remove_unused_columns=True,\n",
       "report_to=['tensorboard'],\n",
       "restore_callback_states_from_checkpoint=False,\n",
       "resume_from_checkpoint=None,\n",
       "run_name=./checkpoints,\n",
       "save_on_each_node=False,\n",
       "save_only_model=False,\n",
       "save_safetensors=True,\n",
       "save_steps=1000,\n",
       "save_strategy=IntervalStrategy.EPOCH,\n",
       "save_total_limit=3,\n",
       "seed=42,\n",
       "skip_memory_metrics=True,\n",
       "split_batches=None,\n",
       "tf32=None,\n",
       "torch_compile=False,\n",
       "torch_compile_backend=None,\n",
       "torch_compile_mode=None,\n",
       "torch_empty_cache_steps=None,\n",
       "torchdynamo=None,\n",
       "tpu_metrics_debug=False,\n",
       "tpu_num_cores=None,\n",
       "use_cpu=False,\n",
       "use_ipex=False,\n",
       "use_legacy_prediction_loop=False,\n",
       "use_mps_device=False,\n",
       "warmup_ratio=0.0,\n",
       "warmup_steps=0,\n",
       "weight_decay=0.01,\n",
       ")"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 209
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step8 创建Trainer"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:21:40.545724Z",
     "start_time": "2025-09-07T11:21:40.527791Z"
    }
   },
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "\n",
    "# *** 参数冻结 ***\n",
    "for name, param in model.bert.named_parameters():\n",
    "\tparam.requires_grad = False\n",
    "\n",
    "trainer = Trainer(model = model,\n",
    "\targs = train_args,\n",
    "\ttokenizer = tokenizer,\n",
    "\ttrain_dataset = tokenized_datasets[\"train\"],\n",
    "\teval_dataset = tokenized_datasets[\"test\"],\n",
    "\tdata_collator = DataCollatorWithPadding(tokenizer = tokenizer),\n",
    "\tcompute_metrics = eval_metric)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86134\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\accelerate\\accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    }
   ],
   "execution_count": 207
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step9 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-07T11:22:05.949040Z",
     "start_time": "2025-09-07T11:21:43.274216Z"
    }
   },
   "source": "trainer.train()",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86134\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\torch\\_dynamo\\eval_frame.py:838: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.5 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  return fn(*args, **kwargs)\n",
      "C:\\Users\\86134\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\torch\\utils\\checkpoint.py:86: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:20, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.653726</td>\n",
       "      <td>0.693694</td>\n",
       "      <td>0.814930</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": "04f1606c14d8bf574146e96cdc450532"
     }
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=13, training_loss=0.6668219933143029, metrics={'train_runtime': 22.2748, 'train_samples_per_second': 313.717, 'train_steps_per_second': 0.584, 'total_flos': 387683447144448.0, 'train_loss': 0.6668219933143029, 'epoch': 0.9454545454545454})"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 208
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "trainer.evaluate(tokenized_datasets[\"test\"])"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "trainer.predict(tokenized_datasets[\"test\"])"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step10 模型预测"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "sen = \"我觉得这家酒店不错，饭很好吃！\"\n",
    "id2_label = {0: \"差评！\", 1: \"好评！\"}\n",
    "model.eval()\n",
    "with torch.inference_mode():\n",
    "\tinputs = tokenizer(sen, return_tensors = \"pt\")\n",
    "\tinputs = {k: v.cuda() for k, v in inputs.items()}\n",
    "\tlogits = model(**inputs).logits\n",
    "\tpred = torch.argmax(logits, dim = -1)\n",
    "\tprint(f\"输入：{sen}\\n模型预测结果:{id2_label.get(pred.item())}\")"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "\n",
    "model.config.id2label = id2_label\n",
    "pipe = pipeline(\"text-classification\", model = model, tokenizer = tokenizer, device = 0)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "pipe(sen)"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
